'''
__author__ = 'xxz'
__email__ = 'talos.tse@gmail.com'
Created on '2013/12/04 17:22'
'''

import sgmllib
import urllib
import Queue
import re
import time


class URLParser(sgmllib.SGMLParser):
    def setbaseurl(self, baseurl):
        self.baseurl = baseurl

    def reset(self):
        sgmllib.SGMLParser.reset(self)
        self.URLqueue = Queue.Queue(maxsize = -1)

    def start_a(self, attrs):
        href = [v for k, v in attrs if k == 'href']
        if href:
            
            for u in href:
                pat = re.compile(r'http://(.+?)')
                if len(re.findall(pat, u)) == 0:
                    continue
                self.URLqueue.put(u) 
            
            if href[0][0] == r'.' or href[0][0] == r'/':
                         
                url=self.baseurl+href[0]
                self.URLqueue.put(url)


def geturlist(url, URLParser):
    try:
        pagedate = urllib.urlopen(url)        
        URLParser.setbaseurl(url)
        URLParser.feed(pagedate.read())
        pagedate.close()
    except:
        #print('[%s]ERROR:%s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), url))
        return

startURL = "http://images.baidu.com"
urlsn=0
myURLParser = URLParser(startURL)
geturlist(startURL, myURLParser)


try:
    while myURLParser.URLqueue.empty() == False:
        urlsn = urlsn + 1
        url = myURLParser.URLqueue.get()
        print("%d:[%s]" % (urlsn, url))
        print("QSize %d" % myURLParser.URLqueue.qsize())
        geturlist(url, myURLParser)
finally:
    myURLParser.close()